# -*- coding:utf-8 -*-
from scrapy.selector import Selector
from spidertools.utils.xpath_utils import get_alltext, get_all_tables
from spidertools.utils.text_utils import replace_punctuation
from spidertools.utils.snippets import combine_two_dict
from info_fsm import InfoMachine
import sys
from pprint import pprint
import requests
from table_info_extract.extract_utils import table_info_extract_styletwo, table_info_extract_stylethree
from table_info_extract import dict_mapping_triedTree
from table_info_extract.common_table_utils import common_table_extrator
from extractors.base_extractor import BaseExtractor
import os
import json
import re
from lxml import etree
from table_info_extract.table_fsm import TableExtractMachine
from utils.logger_utils import savelog, LogStateString
from spidertools.utils.time_utils import get_standardize_time



class JiangSuShengGongLuShuiLuJianSheShiChangXinYongXinXiFuWuXiTong(BaseExtractor):

    def html_replace_word(self, html): #网页源码输入时即进行处理
        html = html.replace('：',':')
        html = html.replace('（', '(')
        html = html.replace('）', ')')
        if '保险经纪人' in html:
            html = html.replace('保险经纪人', '保险经纪人:')
        no_need = ['网址', '联系人', '地址', '传真', '联系电话', '电话', '邮箱']

        for i in no_need:
            a = '(' + i
            if a  in html:
                html = html.replace(a, '不要')

        a_list = ['邮箱', '联系方式', '联系人']
        first_list = [';', '，', '。']
        second_list = [':', '：']

        for a in a_list:
            for first in first_list:
                for second in second_list:
                    nono = first + a + second
                    if nono in html:
                        html = html.replace(nono, '不要:')
        return html

    def check_purchase(self):
        return_flag = False
        if '采购' in self.info_dict['公告标题']:
            return_flag = True
        return return_flag

    def start_parse(self):
        if self.check_purchase():
            savelog(LogStateString.Waring_NoNeedParse, msg_dict={"url": self.info_dict["origin_url"]})
            return {"not_need_checkjson": '1'}

        if self.info_dict['公告类型'] == '中标公告':
            if '标段一：' in self.html:
                self.html = self.html.replace('标段一：', '')
                self.sel = Selector(text=self.html)
            table_extractor = TableExtractMachine()
            table = self.sel.xpath('//*[@id="body"]/div/table')
            result = table_extractor.run_table(table)
            return result
        else:
            # if '跳转前网页' in self.html:  # 如果是json格式的，用json解析（仅供本地测试使用）
            if self.html.startswith('{'):  # 传入的数据若以{为开头，则为json格式的网页
                self.html = self.html_replace_word(self.html)
                result = self.new_json_parse(self.html)
                html_info_dict = dict_mapping_triedTree(self.info_dict)
                if result:  # 如果网页解析返回值正常
                    result = combine_two_dict(result, html_info_dict)
                    return result
                else:
                    print('网页出错，格式不同')
            else:
                result = self.new_commom_announcement_parse()  # 调用表格解析模块
                html_info_dict = dict_mapping_triedTree(self.info_dict)
                result = combine_two_dict(result, html_info_dict)
                return result




    def new_commom_announcement_parse(self):
        self.html = self.html_replace_word(self.html)
        self.sel = Selector(text=self.html)
        output_dict = {}
        tables = []
        qita_text=''
        # 获取所有的table表格
        content_root_nodes = self.sel.xpath("//div[@class='tips noline']")
        for node in content_root_nodes:
            node_text = str(node.xpath('string(.)')[0])
            if '、预约' in node_text:
                continue
            elif '、招标投标文件' in node_text:
                continue
            elif '、监管单位' in node_text:
                continue
            elif '其它说明' in node_text:
                qita_text = node_text
            else:
                table_nodes = node.xpath(".//table")
                tables.extend(table_nodes)  # extend加一整个列表

        machine = InfoMachine(self.base_pattern, extend_keywords=self.extend_keywords)
        # 执行状态机，解析整个文本

        if qita_text:
            qita_dict = machine.run_list(qita_text)
            output_dict = combine_two_dict(output_dict, qita_dict)
        # table_extractor = TableExtractMachine()


        for  table in tables:
            result = self.table_extract_machine.run_table(table)
            output_dict = combine_two_dict(output_dict, result)
        html_info_dict = dict_mapping_triedTree(self.info_dict)
        output_dict = combine_two_dict(output_dict, html_info_dict)
        if '工程招标信息' not in output_dict.keys():
            output_dict['工程招标信息'] = {}
        try:
            if '标段名称' not in output_dict['工程招标信息'].keys():
                output_dict['工程招标信息']['标段名称'] = output_dict['工程基本信息']['项目名称']
        except Exception as a:
            pass

        try:  # 防止多匹配
            a = ['a']
            if len(output_dict['未知']['未知电话']) > 1:
                a[0] = output_dict['未知']['未知电话'][0]
                output_dict['未知']['未知电话'] = a
        except Exception as f:
            pass


        if ':' in output_dict['工程招标信息']['招标内容']: #将招标内容中的文本再走一遍文本解析
            neirong_list=['neirong']
            neirong_list[0]=output_dict['工程招标信息']['招标内容']
            self.base_pattern = '%s:\s*(.*?)。'
            self.texts_extract_machine = InfoMachine(self.base_pattern, extend_keywords=self.extend_keywords)
            output_dict_2=self.texts_matchine_parse(neirong_list)
            try:
                del output_dict_2['工程招标信息']['招标内容']
            except Exception as a:
                pass
            output_dict = combine_two_dict(output_dict, output_dict_2)
        try:
            output_dict['工程招标信息']["投标截止时间"] = get_standardize_time(output_dict['工程招标信息']["投标截止时间"])
        except Exception as a:
            pass
        output_dict = weizhi(output_dict)
        output_dict = re_juhao(output_dict)
        return output_dict

    def new_json_parse(self, response):  # 用于解析json格式的网页
        h = re.compile(r'[{](.*?)"[}]', re.S)  # 定义提取大括号内内容的正则
        response = re.findall(h, response)
        response_list = list(response[0])
        response_list.append('"')
        response = ''.join(response_list)
        response = '{' + response + '}'
        response = rf"""{response}"""
        try:  # 如果json网页能正常转换
            response = json.loads(response)  # 转换成字典格式
            html = response['TENDERCONTENT']  # 得到指定的标签
            html = '<html> <body> ' + html + ' </body> </html>'
            html = rf"""{html}"""
            self.sel = Selector(text=html)
            output_dict_text = self.common_text_parse(xpath='//body/*')#文本解析部分
            # 表格解析
            tables = get_all_tables(self.sel)
            output_dict ={}
            for table in tables:
                result = self.table_extract_machine.run_table(table)
                output_dict = combine_two_dict(output_dict, result)
            output_dict = combine_two_dict(output_dict, output_dict_text)
            html_info_dict = dict_mapping_triedTree(self.info_dict)
            output_dict = combine_two_dict(output_dict, html_info_dict)

            # 最后加工
            if '工程基本信息' not in output_dict.keys():
                output_dict['工程基本信息'] = {}
            if '工程招标信息' not in output_dict.keys():
                output_dict['工程招标信息'] = {}
            output_dict['工程基本信息']['项目名称'] = find_project_name(response['BNNAME'])
            output_dict['工程招标信息']['标段名称'] = response['PARAGRAPHNAME']
            output_dict['公告标题'] = response['BNNAME']
            try:  # 防止多匹配
                a = ['a']
                if len(output_dict['工程招标信息']['招标代理机构地址']) > 1:
                    a[0] = output_dict['工程招标信息']['招标代理机构地址'][0]
                    output_dict['工程招标信息']['招标代理机构地址'] = a
            except Exception as f:
                pass
            try:  # 防止多匹配
                b = ['b']
                if len(output_dict['工程招标信息']['招标代理机构电话']) > 1:
                    b[0] = output_dict['工程招标信息']['招标代理机构电话'][0]
                    output_dict['工程招标信息']['招标代理机构电话'] = b
            except Exception as f:
                pass
            output_dict = check_repeat(output_dict)
            output_dict = re_juhao(output_dict)
            try:
                if output_dict['工程招标信息']['标段名称'] == '':
                    output_dict['工程招标信息']['标段名称'] = re.findall('[\(（][^\)）]+[\)）]$', output_dict['工程基本信息']['项目名称'])[0]
            except Exception as e:
                pass

            return output_dict
        except Exception as wrong:  # 网页无法转换，直接跳过
            print(wrong)
    def common_replace_word(self, text):
        text = text.replace('&nbsp;', '')  # 去除所有的&nbsp;
        text = text.replace('\u3000', '')
        text = text.replace('\xa0', '')
        text = text.replace('\t', '')
        return text


def weizhi(output_dict):  # 用于解决未知的联系人或电话中包含代理的情况，拆分分别赋予
    try:
        for i in output_dict['未知']['未知电话']:
            if '代理' in i:
                x = i.split('、', 1)
                for a in x:
                    if '代理' in a:
                        z = ['z']
                        a = re.sub(u"\\(.*?\\)", "", a)
                        z[0] = a
                        output_dict["工程招标信息"]["招标代理机构电话"] = z
                    else:
                        z = ['z']
                        a = re.sub(u"\\(.*?\\)", "", a)
                        z[0] = a
                        output_dict["工程招标信息"]["招标人信息电话"] = z
                del output_dict['未知']['未知电话']
            else:
                pass

        for q in output_dict['未知']['未知联系人']:
            if '代理' in q:
                x = q.split('、', 1)
                for q in x:
                    if '代理' in q:
                        z = ['z']
                        q = re.sub(u"\\(.*?\\)", "", q)
                        z[0] = q
                        output_dict["工程招标信息"]["招标代理机构联系人"] = z
                    else:
                        z = ['z']
                        q = re.sub(u"\\(.*?\\)", "", q)
                        z[0] = q
                        output_dict["工程招标信息"]["招标人信息联系人"] = z
                del output_dict['未知']['未知联系人']
            else:
                pass
        for q in output_dict['未知']['未知传真']:
            if '代理' in q and '、' in q:
                x = q.split('、', 1)
                for q in x:
                    if '代理' in q:
                        z = ['z']
                        q = re.sub(u"\\(.*?\\)", "", q)#去除括号内内容
                        z[0] = q
                        output_dict["工程招标信息"]["招标代理机构传真"] = z
                    else:
                        z = ['z']
                        q = re.sub(u"\\(.*?\\)", "", q)
                        z[0] = q
                        output_dict["工程招标信息"]["招标人信息传真"] = z
                del output_dict['未知']['未知联系人']
            elif '代理' in q and '、' not in q:
                z = ['z']
                q = re.sub(u"\\(.*?\\)", "", q)
                z[0] = q
                output_dict["工程招标信息"]["招标代理机构传真"] = z
                del output_dict['未知']['未知传真']
            elif '代理' not in q and '、' in q:
                x = q.split('、', 1)
                output_dict["工程招标信息"]["招标人信息传真"] = x[0]
                output_dict["工程招标信息"]["招标代理机构传真"] = x[1]
                del output_dict['未知']['未知传真']
            else:
                pass
    except Exception as q:
        pass
    return output_dict




def find_project_name(gonggao_name):  # 在取得项目名称时，遇如下非工程关键字，则保留项目完整名称
    if '招标公告' in gonggao_name:
        if '项目' in gonggao_name and '工程' in gonggao_name:
            project_name = re.findall("(.*?)项目", gonggao_name)[0] + '项目'
        elif '项目' in gonggao_name and '工程' not in gonggao_name:
            project_name = re.findall("(.*?)项目", gonggao_name)[0] + '项目'
        elif '项目' not in gonggao_name and '工程' in gonggao_name:
            project_name = re.findall("(.*?)工程", gonggao_name)[0] + '工程'
        else:
            project_name = gonggao_name.replace('招标公告', '')
    elif '招标公告' not in gonggao_name:
        if '项目' in gonggao_name and '工程' in gonggao_name:
            project_name = re.findall("(.*?)项目", gonggao_name)[0] + '项目'
        elif '项目' in gonggao_name and '工程' not in gonggao_name:
            project_name = re.findall("(.*?)项目", gonggao_name)[0] + '项目'
        elif '项目' not in gonggao_name and '工程' in gonggao_name:
            project_name = re.findall("(.*?)工程", gonggao_name)[0] + '工程'
        else:
            project_name = gonggao_name
    return project_name



def check_repeat(output_dict):  # 在输出前删除列表中重复的数据
    realfirstlist = []
    realsecondlist = []
    for k, v in output_dict.items():
        if type(v) == dict:
            realfirstlist.append(k)
            realsecondlist.append([i for i in output_dict[k].keys()])
    for k, v in zip(realfirstlist, realsecondlist):
        if len(v) != 0:
            ##一级目录为空
            for i in v:
                content = output_dict[k][i]
                if type(content) == list:
                    if len(content) > 1:
                        if content[0] == content[1]:
                            del content[1]
                            output_dict[k][i] = content
                else:
                    pass
    return output_dict




def re_juhao(output_dict):
    try:
        if '。' in output_dict["工程基本信息"]["工程基本信息工期"] and len(output_dict["工程基本信息"]["工程基本信息工期"]) > 100:
            output_dict["工程基本信息"]["工程基本信息工期"] = re.findall('(.*?)。', output_dict["工程基本信息"]["工程基本信息工期"])[0]
    except Exception as a:
        pass
    try:
        if '。' in output_dict["工程招标信息"]["招标人信息工期"] and len(output_dict["工程招标信息"]["招标人信息工期"]) > 100:
            output_dict["工程招标信息"]["招标人信息工期"] = re.findall('(.*?)。', output_dict["工程招标信息"]["招标人信息工期"])[0]
    except Exception as a:
        pass
    try:
        if '。' in output_dict["工程基本信息"]["建设周期"] and len(output_dict["工程基本信息"]["建设周期"]) > 100:
            output_dict["工程基本信息"]["建设周期"] = re.findall('(.*?)。', output_dict["工程基本信息"]["建设周期"])[0]
    except Exception as a:
        pass
    try:
        if '。' in output_dict["工程基本信息"]["工程基本信息名称"] and len(output_dict["工程基本信息"]["工程基本信息名称"]) > 100:
            output_dict["工程基本信息"]["工程基本信息名称"] = re.findall('(.*?)。', output_dict["工程基本信息"]["工程基本信息名称"])[0]

    except Exception as a:
        pass
    try:
        if '。' in output_dict["工程招标信息"]["标段工期"] and len(output_dict["工程招标信息"]["标段工期"]) > 100:
            output_dict["工程招标信息"]["标段工期"] = re.findall('(.*?)。', output_dict["工程招标信息"]["标段工期"])[0]
    except Exception as a:
        pass
    try:
        if '。' in output_dict["工程基本信息"]["工程基本信息名称"] and len(output_dict["工程基本信息"]["工程基本信息名称"]) > 100:
            output_dict["工程基本信息"]["工程基本信息名称"] = re.findall('(.*?)。', output_dict["工程基本信息"]["工程基本信息名称"])[0]
    except Exception as a:
        pass
    try:
        if '。' in output_dict["工程招标信息"]["招标人信息地址"] and len(output_dict["工程招标信息"]["招标人信息地址"]) > 100:
            output_dict["工程招标信息"]["招标人信息地址"] = re.findall('(.*?)。', output_dict["工程招标信息"]["招标人信息地址"])[0]
    except Exception as a:
        pass
    try:
        if '。' in output_dict["工程基本信息"]["工程地址"] and len(output_dict["工程基本信息"]["工程地址"]) > 100:
            output_dict["工程基本信息"]["工程地址"] = re.findall('(.*?)。', output_dict["工程基本信息"]["工程地址"])[0]
    except Exception as a:
        pass
    try:
        if '。' in output_dict["工程基本信息"]["项目名称"] and len(output_dict["工程基本信息"]["项目名称"]) > 100:
            output_dict["工程基本信息"]["项目名称"] = re.findall('(.*?)。', output_dict["工程基本信息"]["项目名称"])[0]
    except Exception as a:
        pass
    return output_dict


if __name__ == '__main__':
    info_dict = {}
    for root, dirs, files, in os.walk('D:/2.19/company_info_extrator/demo_html/shuili'):
        for i, name in enumerate(files):
            if "init" not in name:
                print(f"\033[32m{i + 1}=============={name}\033[0m")
                with open(f'D:/2.19/company_info_extrator/demo_html/shuili/{name}', 'r',
                          encoding='utf-8') as fr:
                    html = fr.read()
                info_dict['html'] = html
                info_dict['公告类型'] = "招标公告"  # "中标结果公告"
                info_dict["source_type"] = "江苏省公路水路建设市场信用信息服务系统"
                info_dict['province'] = "江苏"
                info_dict['_id'] = "1"
                info_dict["origin_url"] = '1'
                # info_dict['公告标题'] ='采购个球'
                obj = JiangSuShengGongLuShuiLuJianSheShiChangXinYongXinXiFuWuXiTong(info_dict)
                result = obj.output_extractor_dict()
                pprint(result)
